﻿using System;
using System.Net;
using System.IO;
using System.Collections.Generic;
using System.Text.RegularExpressions;
using MySql.Data.MySqlClient;
using System.Threading;
using System.Data;
using HouseObj;

namespace ScrawlLianjiaData.Server
{
    /// <summary>
    /// //链家网数据爬虫对象
    /// </summary>
    class WebCrawler
    {
        #region Members
        /// <summary>
        /// //存储链家网上有数据城市和城市房价信息网站链接的文件地址
        /// </summary>
        private static Dictionary<string, List<string>> cityInfo = new Dictionary<string, List<string>>();
        #endregion

        #region Attributes
        /// <summary>
        /// //数据库连接字符串
        /// </summary>
        public string mySqlConText { get; set; }
        /// <summary>
        /// //数据库名称
        /// </summary>
        public string myDatabase { get; set; }
        #endregion

        #region Constructors
        public void ReadConfiguration()
        {
            StringReader ssr = new StringReader(Properties.Resources.city);   //读取文件并按行(城市)隔开
            string[] cityMsgs = ssr.ReadToEnd().Replace(" ", "").Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
            for (int i = 0; i < cityMsgs.Length; i++)
            {
                string[] tpMsg = cityMsgs[i].Split('\t');       //依次存入每一个城市的名称和响应的网址链接
                List<string> tpInfo = new List<string>();
                for (int j = 1; j < tpMsg.Length; j++)
                    tpInfo.Add(tpMsg[j]);
                cityInfo.Add(tpMsg[0], tpInfo);
            }
            ssr.Close();            //释放资源
            ssr.Dispose();
        }
        public void SetDatabase()
        {
            MySqlConnection mySqlCon = new MySqlConnection(mySqlConText);
            mySqlCon.Open();                //打开与数据库的连接
            MySqlDataAdapter mySqlAdapter = new MySqlDataAdapter("SHOW TABLES FROM "+ myDatabase, mySqlCon);        //获取数据库中表的数目，决定要不要新建表
            DataSet ds = new DataSet();
            mySqlAdapter.Fill(ds, "TABLES");
            mySqlAdapter.Dispose();
            DataTable dt = ds.Tables["TABLES"];
            MySqlCommand mySqlCommand = new MySqlCommand();
            mySqlCommand.Connection = mySqlCon;
            string commandText = string.Empty;
            if (dt.Rows.Count < 1)        //表数小于1，新建二手房表
                commandText += "CREATE TABLE `SECONDHOUSE` (`DATAID` TINYTEXT,`NAME` TINYTEXT,`CITY` TINYTEXT,`REGION` TINYTEXT,`DISCRIPTION` MEDIUMTEXT,`REALMAPURL` MEDIUMTEXT,`LAYOUTMAPURL` MEDIUMTEXT,`DETAILURL` MEDIUMTEXT,`CONFIGURATION` MEDIUMTEXT,`AREA` DOUBLE,`PRICE` DOUBLE,`PRICEMSG` TINYTEXT,`LOOKEDPEOPLENUM` INT,`URL` MEDIUMTEXT);";
            if (dt.Rows.Count < 2)        //表数小于2，新建楼盘表
                commandText += "CREATE TABLE `SELLINGHOUSE` (`NAME` TINYTEXT,`CITY` TINYTEXT,`REGION` TINYTEXT,`REALMAPURL` MEDIUMTEXT,`DETAILURL` MEDIUMTEXT,`ADDRESS` MEDIUMTEXT,`CONFIGURATION` MEDIUMTEXT,`AREA` TINYTEXT,`ONSOLD` TINYTEXT,`LIVETYPE` TINYTEXT,`AVEPRICE` DOUBLE,`URL` MEDIUMTEXT);";
            if (dt.Rows.Count < 3)        //表数小于3，新建租房表
                commandText += "CREATE TABLE `RENTINGHOUSE` (`DATAID` TINYTEXT,`NAME` TINYTEXT,`CITY` TINYTEXT,`REGION` TINYTEXT,`DISCRIPTION` MEDIUMTEXT,`REALMAPURL` MEDIUMTEXT,`LAYOUTMAPURL` MEDIUMTEXT,`DETAILURL` MEDIUMTEXT,`CONFIGURATION` MEDIUMTEXT,`AREA` DOUBLE,`PRICE` DOUBLE,`PRICEMSG` TINYTEXT,`LOOKEDPEOPLENUM` INT,`URL` MEDIUMTEXT);";
            if (dt.Rows.Count < 4)        //表数小于4，新建小区表
                commandText += "CREATE TABLE `HOODHOUSE` (`NAME` TINYTEXT,`CITY` TINYTEXT,`REGION` TINYTEXT,`AVEPRICE` DOUBLE,`HOUSENUM` INT,`REALMAPURL` MEDIUMTEXT,`DETAILURL` MEDIUMTEXT,`URL` MEDIUMTEXT);";
            commandText += "DELETE FROM SECONDHOUSE;DELETE FROM SELLINGHOUSE;DELETE FROM RENTINGHOUSE;DELETE FROM HOODHOUSE;";
            mySqlCommand.CommandText = commandText;
            mySqlCommand.ExecuteNonQuery();
            mySqlCommand.Dispose();   //释放所有资源
            dt.Dispose();
            ds.Dispose();
            mySqlCon.Close();
            mySqlCon.Dispose();
        }
        #endregion

        #region PublicMethods
        /// <summary>
        /// //开始爬取数据
        /// </summary>
        public void Start()
        {
            Thread.CurrentThread.Priority = ThreadPriority.Highest;
            foreach (string key in cityInfo.Keys)
            {
                Thread thread1 = new Thread(new ParameterizedThreadStart(StartCrawlData));
                thread1.Priority = ThreadPriority.AboveNormal;
                thread1.Start(new ThreadHousePara(key, cityInfo[key][0], cityInfo[key][1], HouseDataType.SecondHouse));
                Thread thread2 = new Thread(new ParameterizedThreadStart(StartCrawlData));
                thread2.Priority = ThreadPriority.Lowest;
                thread2.Start(new ThreadHousePara(key, cityInfo[key][0], cityInfo[key][2], HouseDataType.SellingHouse));
                Thread thread3 = new Thread(new ParameterizedThreadStart(StartCrawlData));
                thread3.Priority = ThreadPriority.Lowest;
                thread3.Start(new ThreadHousePara(key, cityInfo[key][0], cityInfo[key][3], HouseDataType.RentingHouse));
                Thread thread4 = new Thread(new ParameterizedThreadStart(StartCrawlData));
                thread4.Priority = ThreadPriority.Lowest;
                thread4.Start(new ThreadHousePara(key, cityInfo[key][0], cityInfo[key][4], HouseDataType.HoodHouse));
            }
        }
        #endregion

        #region CrawlMethods
        /// <summary>
        /// //线程函数，用来下载指定城市指定房屋类型的数据
        /// </summary>
        /// <param name="obj"></param>
        private void StartCrawlData(object obj)
        {
            string cityName = ((ThreadHousePara)obj).cityName;
            string city = ((ThreadHousePara)obj).city;
            string server = ((ThreadHousePara)obj).server;
            HouseDataType housetype = ((ThreadHousePara)obj).houseType;
            if (server != string.Empty)
            {
                Dictionary<string,string> urls = new Dictionary<string, string>();       //获取要爬取的网页地址
                if (housetype == HouseDataType.SellingHouse)
                    urls.Add(@"http://" + server, "");
                else
                    urls = CrawlCityRegionUrl(city, server);
                foreach (string key in urls.Keys)        //对一个网页地址爬取数据
                {
                    string url = key;
                    while (true)
                    {
                        if (url == null)          //如果网址无效(说明已经到最后一页)，则退出循环
                            break;
                        string content = GetUrlHtmlContent(url);          //获取网页内容
                        bool hasNoFault = true;
                        if (housetype == HouseDataType.SecondHouse)            //爬取数据
                            hasNoFault = CrawlSecondHousePriceData(url, cityName, urls[key], content);
                        else if (housetype == HouseDataType.SellingHouse)
                            hasNoFault = CrawSellingHousePriceData(url, cityName, urls[key], content);
                        else if (housetype == HouseDataType.RentingHouse)
                            hasNoFault = CrawlRentingHousePriceData(url, cityName, urls[key], content);
                        else if (housetype == HouseDataType.HoodHouse)
                            hasNoFault = CrawlHoodHousePriceData(url, cityName, urls[key], content);
                        if (hasNoFault)                         //获取下一页网页地址
                            url = GrawlNextPageUrl(city, content, housetype);
                    }
                    Console.WriteLine(DateTime.Now.ToString("yyyy-MM-dd-HH-mm-ss") + " " + key + " " + cityName + " " + housetype.ToString());
                }
            }
        }
        /// <summary>
        /// //获取城市区块服务器地址
        /// </summary>
        /// <param name="city">城市代码</param>
        /// <param name="server">数据服务器</param>
        /// <returns>区块服务器地址集合</returns>
        private Dictionary<string,string> CrawlCityRegionUrl(string city, string server)
        {
            Dictionary<string,string> urls = new Dictionary<string, string>();
            string htmlContent = GetUrlHtmlContent(@"http://" + server);   //获取城市网页内容并匹配区县网址
            MatchCollection options = Regex.Matches(Regex.Match(htmlContent, "<dl class=\"dl-lst clear\".*?</dl>").Value, "<a href.*?</a>");
            for (int i = 0; i < options.Count; i++)
            {
                string option = options[i].Value;
                if (option.IndexOf("不限") == -1)
                {
                    string regionName = Regex.Match(option, ">.*?</a>").Value;
                    regionName = regionName.Substring(1, regionName.Length - 5);
                    string regionUrl = Regex.Match(option, "href=\".*?\"").Value;
                    string optionHtmlContent = GetUrlHtmlContent(@"http://" + city + ".lianjia.com" + regionUrl.Substring(6, regionUrl.Length - 7));
                    optionHtmlContent = Regex.Match(optionHtmlContent, "<div class=\"option-list sub-option-list\">.*?</div>").Value;
                    MatchCollection hoods = Regex.Matches(optionHtmlContent, "<a href.*?</a>");    //获取二级选项
                    for (int j = 0; j < hoods.Count; j++)
                    {
                        string hood = hoods[j].Value;
                        if (hood.IndexOf("不限") == -1)            //添加城市区县各区块的数据网址
                        {
                            string hoodUrl = Regex.Match(hood, "href=\".*?\"").Value;
                            hoodUrl = @"http://" + city + ".lianjia.com" + hoodUrl.Substring(6, hoodUrl.Length - 7);
                            string hoodName = Regex.Match(hood, ">.*?</a>").Value;
                            hoodName = hoodName.Substring(1, hoodName.Length - 5);
                            if (!urls.ContainsKey(hoodUrl))
                                urls.Add(hoodUrl, regionName + " " + hoodName);
                        }
                    }
                }
            }
            return urls;
        }
        /// <summary>
        /// //获取下一页网址
        /// </summary>
        /// <param name="city">城市</param>
        /// <param name="content">网页内容</param>
        /// <param name="housetype">数据类型</param>
        /// <returns>下一页网址，没有下一页返回NULL</returns>
        private string GrawlNextPageUrl(string city, string content, HouseDataType housetype)
        {
            string server = housetype == HouseDataType.SellingHouse ? ".fang.lianjia.com" : ".lianjia.com";   //设置服务器地址
            content = Regex.Match(content, "<div class=\"page-box house-lst-page-box\".*?</div>").Value;
            if (content == "") return null;            //网址内容有误
            MatchCollection pageCol = Regex.Matches(content, "totalPage.*?curPage.*?>");   //处理两种网页结构(上海式的和北京式的)
            if (pageCol.Count != 0)       //北京式的网页结构
            {
                MatchCollection numCol = Regex.Matches(pageCol[0].Value, "(\\d)+");
                int totalPageNum = Convert.ToInt32(numCol[0].Value);            //获取当前页码和全部页码
                int curPageNum = Convert.ToInt32(numCol[1].Value);
                if (curPageNum < totalPageNum)
                {
                    string url = Regex.Match(content, "page-url=\".*?\"").Value;        //获取下一页地址
                    return @"http://" + city + server + url.Substring(10, url.Length - 11).Replace("{page}", (curPageNum + 1).ToString());
                }
            }
            else
            {
                MatchCollection linkCol = Regex.Matches(content, "<a.+?</a>");      //获取底部页码的所有连接
                if (linkCol.Count != 0 && linkCol[linkCol.Count - 1].Value.IndexOf("class=\"on\"") == -1)   //如果有页码而且最后一个页码不是当前页
                {
                    string url = Regex.Match(linkCol[linkCol.Count - 1].Value, "href=\".*?\"").Value;   //获取下一页按钮链接的地址
                    return url == "" ? null : @"http://" + city + server + url.Substring(6, url.Length - 7);
                }
            }
            return null;
        }
        /// <summary>
        /// //获取二手房房价信息
        /// </summary>
        /// <param name="cityName">城市</param>
        /// <param name="content">回应网址内容</param>
        /// <return>True 成功获取数据，False 出错</return>
        private bool CrawlSecondHousePriceData(string url, string cityName, string region, string content)
        {
            try
            {
                List<MyHouseInfo> secHouseCol = new List<MyHouseInfo>();
                string houseInfo = Regex.Match(content, "<ul id=\"house-lst\" class=\"house-lst\".*?</ul>").Value;        //正则表达式分析
                string[] houseItems = Regex.Match(houseInfo, "<li.*?</ul>").Value.Replace("</ul>", "").Split(new string[] { "</li>" }, StringSplitOptions.RemoveEmptyEntries);
                for (int i = 0; i < houseItems.Length; i++)
                {
                    if (houseItems[i].Trim() == "" || houseItems[i].Trim().IndexOf("没有找到") != -1)
                        continue;
                    string dataId = Regex.Match(houseItems[i], "data-id=\".*?\"").Value;
                    if (dataId != "")
                        dataId = dataId.Substring(9, dataId.Length - 10);
                    else
                    {
                        dataId = Regex.Match(houseItems[i], "key=\".*?\"").Value;
                        dataId = dataId == "" ? dataId : dataId.Substring(5, dataId.Length - 6);
                    }
                    string realMapUrl = Regex.Match(houseItems[i], "data-img-real=\".*?\"").Value;
                    if (realMapUrl != "")
                        realMapUrl = realMapUrl.Substring(15, realMapUrl.Length - 16);
                    else
                    {
                        realMapUrl = Regex.Match(houseItems[i], "data-img=\".*?\"").Value;
                        realMapUrl = realMapUrl == "" ? realMapUrl : realMapUrl.Substring(10, realMapUrl.Length - 11);
                    }
                    string layoutMapUrl = Regex.Match(houseItems[i], "data-apart-layout=\".*?\"").Value;
                    if (layoutMapUrl != "")
                        layoutMapUrl = layoutMapUrl.Substring(19, layoutMapUrl.Length - 20);
                    else
                    {
                        layoutMapUrl = Regex.Match(houseItems[i], "data-img-layout=\".*?\"").Value;
                        layoutMapUrl = layoutMapUrl == "" ? layoutMapUrl : layoutMapUrl.Substring(17, layoutMapUrl.Length - 18);
                    }

                    string discription = Regex.Match(houseItems[i], "title=\".*?\"").Value;
                    discription = discription == "" ? discription : discription.Substring(7, discription.Length - 8);

                    string whereMsg = Regex.Match(houseItems[i], "<div class=\"where\">.*?</div>").Value;
                    string detailUrl = Regex.Match(whereMsg, "href=\".*?\"").Value;
                    detailUrl = detailUrl == "" ? detailUrl : detailUrl.Substring(6, detailUrl.Length - 7);
                    MatchCollection whereInfo = Regex.Matches(whereMsg, "<span.*?</span>");
                    string name = whereInfo[0].Value.Replace("</a>", "");
                    name = name == "" ? name : name.Substring(15, name.Length - 22);
                    string configuration = whereInfo[1].Value.Replace("&nbsp;", "").Replace(" class=\"\"", "");
                    configuration = configuration == "" ? configuration : configuration.Substring(12, configuration.Length - 19);
                    string area = whereInfo[2].Value.Replace("&nbsp;", "");
                    area = Regex.Match(area, "(\\d|\\.)+").Value;

                    string totalPrice = Regex.Match(houseItems[i], "<div class=\"price\"> *<span class=\"num\">.*?</span>").Value;
                    totalPrice = Regex.Match(totalPrice, "(\\d|\\.)+").Value;
                    string price = Regex.Match(houseItems[i], "<div class=\"price-pre\">.*?</div>").Value;
                    price = Regex.Match(price, "(\\d|\\.)+").Value;

                    string lookedPeopleNum = Regex.Match(houseItems[i], "<div class=\"square\">.*?<span class=\"num\">.*?</span>").Value;
                    lookedPeopleNum = Regex.Match(lookedPeopleNum, "(\\d|\\.)+").Value;

                    MyHouseInfo sechouse = new MyHouseInfo(dataId, name, cityName, region, discription, realMapUrl, layoutMapUrl, detailUrl, configuration, area, totalPrice, price, lookedPeopleNum, url);
                    secHouseCol.Add(sechouse);       //新建房屋数据对象
                }
                SaveHouseDataToDB(secHouseCol, "SECONDHOUSE");    //保存数据到数据库，成功返回True
                return true;
            }
            catch (Exception ex)
            {
                Console.WriteLine(DateTime.Now.ToString("yyyy-MM-dd-HH-mm-ss") + " " + ex.Message + " " + url + " " + cityName);
                return false;     //分析出错，放回False，重新再来一遍
            }
        }
        /// <summary>
        /// //获取新售楼盘房价信息
        /// </summary>
        /// <param name="cityName">城市</param>
        /// <param name="content">回应网址内容</param>
        /// <return>True 成功获取数据，False 出错</return>
        private bool CrawSellingHousePriceData(string url, string cityName, string region, string content)
        {
            try
            {
                List<MyBuildingInfo> sellingHouseCol = new List<MyBuildingInfo>();
                string houseInfo = Regex.Match(content, "<ul id=\"house-lst\" class=\"house-lst\".*?</ul> *?<div").Value;//正则表达式分析
                MatchCollection houseItems = Regex.Matches(houseInfo, "<li data-index.+?</li>");
                for (int i = 0; i < houseItems.Count; i++)
                {
                    if (houseItems[i].Value.Trim() == "" || houseItems[i].Value.Trim().IndexOf("没有找到") != -1)
                        continue;
                    string realMapUrl = Regex.Match(houseItems[i].Value, "data-original=\".*?\"").Value;
                    realMapUrl = realMapUrl == "" ? realMapUrl : realMapUrl.Substring(15, realMapUrl.Length - 16);
                    string detailUrl = Regex.Match(houseItems[i].Value, "href=\".*?\"").Value;
                    detailUrl = detailUrl == "" ? detailUrl : detailUrl.Substring(6, detailUrl.Length - 7);

                    string detailMsg = Regex.Match(houseItems[i].Value, "<div class=\"col-1\".*?<div class=\"col-2\"").Value;
                    string name = Regex.Match(detailMsg, "<a.*?</a>").Value;
                    name = Regex.Match(name, ">.*?</a>").Value;
                    name = name == "" ? name : name.Substring(1, name.Length - 5);
                    MatchCollection detailInfo = Regex.Matches(detailMsg.Substring(detailMsg.IndexOf("</h2>")), "<div.*?</div>");
                    string address = Regex.Match(detailInfo[0].Value, "<span class=\"region\">.*?</span>").Value;
                    address = address == "" ? address : address.Substring(21, address.Length - 28);
                    string configuration = Regex.Match(detailInfo[1].Value, "<div class=\"area\">.*?<span>").Value;
                    configuration = configuration == "" ? configuration : configuration.Substring(18, configuration.Length - 24);
                    configuration = configuration.Replace(" ", "");
                    configuration = configuration.Replace("-", "");
                    string area = Regex.Match(detailInfo[1].Value, "<span>.*?</span>").Value;
                    area = area == "" ? area : area.Substring(6, area.Length - 13);
                    string onsold = string.Empty, liveType = string.Empty;
                    if (detailInfo.Count > 3)
                    {
                        onsold = Regex.Match(detailInfo[3].Value, "<span class=\"onsold\">.*?</span>").Value;
                        liveType = Regex.Match(detailInfo[3].Value, "<span class=\"live\">.*?</span>").Value;
                    }
                    else
                    {
                        onsold = Regex.Match(detailInfo[2].Value, "<span class=\"onsold\">.*?</span>").Value;
                        liveType = Regex.Match(detailInfo[2].Value, "<span class=\"live\">.*?</span>").Value;
                    }
                    onsold = onsold == "" ? onsold : onsold.Substring(21, onsold.Length - 28);
                    liveType = liveType == "" ? liveType : liveType.Substring(21, liveType.Length - 28);

                    string avePrice = Regex.Match(houseItems[i].Value, "<div class=\"price\">.*?<span class=\"num\">.*?</span>").Value;
                    avePrice = Regex.Match(avePrice, "(\\d|\\.)+").Value;

                    MyBuildingInfo sellingHouse = new MyBuildingInfo(name, cityName, region, realMapUrl, detailUrl, address, configuration, area, onsold, liveType, avePrice, url);
                    sellingHouseCol.Add(sellingHouse);       //新建楼盘数据对象
                }
                SaveBuildingDataToDB(sellingHouseCol);    //保存数据到数据库，成功返回True
                return true;
            }
            catch (Exception ex)
            {
                Console.WriteLine(DateTime.Now.ToString("yyyy-MM-dd-HH-mm-ss") + " " + ex.Message + " " + url + " " + cityName);
                return false;      //分析出错，放回False，重新再来一遍
            }
        }
        /// <summary>
        /// //获取租房房价信息
        /// </summary>
        /// <param name="cityName">城市</param>
        /// <param name="content">回应网址内容</param>
        ///<return>True 成功获取数据，False 出错</return>
        private bool CrawlRentingHousePriceData(string url, string cityName, string region, string content)
        {
            try
            {
                List<MyHouseInfo> rentHouseCol = new List<MyHouseInfo>();
                string houseInfo = Regex.Match(content, "<ul id=\"house-lst\" class=\"house-lst\".*?</ul>").Value;       //正则表达式分析
                string[] houseItems = Regex.Match(houseInfo, "<li.*?</ul>").Value.Replace("</ul>", "").Split(new string[] { "</li>" }, StringSplitOptions.RemoveEmptyEntries);
                for (int i = 0; i < houseItems.Length; i++)
                {
                    if (houseItems[i].Trim() == "" || houseItems[i].Trim().IndexOf("没有找到") != -1)
                        continue;
                    string dataId = Regex.Match(houseItems[i], "data-id=\".*?\"").Value;
                    if (dataId != "")
                        dataId = dataId.Substring(9, dataId.Length - 10);
                    else
                    {
                        dataId = Regex.Match(houseItems[i], "key=\".*?\"").Value;
                        dataId = dataId == "" ? dataId : dataId.Substring(5, dataId.Length - 6);
                    }
                    string realMapUrl = Regex.Match(houseItems[i], "data-img-real=\".*?\"").Value;
                    if (realMapUrl != "")
                        realMapUrl = realMapUrl.Substring(15, realMapUrl.Length - 16);
                    else
                    {
                        realMapUrl = Regex.Match(houseItems[i], "data-img=\".*?\"").Value;
                        realMapUrl = realMapUrl == "" ? realMapUrl : realMapUrl.Substring(10, realMapUrl.Length - 11);
                    }
                    string layoutMapUrl = Regex.Match(houseItems[i], "data-apart-layout=\".*?\"").Value;
                    if (layoutMapUrl != "")
                        layoutMapUrl = layoutMapUrl.Substring(19, layoutMapUrl.Length - 20);
                    else
                    {
                        layoutMapUrl = Regex.Match(houseItems[i], "data-img-layout=\".*?\"").Value;
                        layoutMapUrl = layoutMapUrl == "" ? layoutMapUrl : layoutMapUrl.Substring(17, layoutMapUrl.Length - 18);
                    }

                    string discription = Regex.Match(houseItems[i], "title=\".*?\"").Value;
                    discription = discription == "" ? discription : discription.Substring(7, discription.Length - 8);

                    string whereMsg = Regex.Match(houseItems[i], "<div class=\"where\">.*?</div>").Value;
                    string detailUrl = Regex.Match(whereMsg, "href=\".*?\"").Value;
                    detailUrl = detailUrl == "" ? detailUrl : detailUrl.Substring(6, detailUrl.Length - 7);
                    MatchCollection whereInfo = Regex.Matches(whereMsg, "<span.*?</span>");
                    string name = whereInfo[0].Value.Replace("</a>", "").Replace("&nbsp;", "");
                    name = name == "" ? name : name.Substring(21, name.Length - 28);
                    string configuration = whereInfo[1].Value.Replace("&nbsp;", "").Replace(" class=\"\"", "");
                    configuration = configuration == "" ? configuration : configuration.Substring(25, configuration.Length - 32);
                    string area = whereInfo[2].Value.Replace("&nbsp;", "");
                    area = Regex.Match(area, "(\\d|\\.)+").Value;

                    string totalPrice = Regex.Match(houseItems[i], "<div class=\"price\"> *<span class=\"num\">.*?</span>").Value;
                    totalPrice = Regex.Match(totalPrice, "(\\d|\\.)+").Value;
                    string price = Regex.Match(houseItems[i], "<div class=\"price-pre\">.*?</div>").Value;
                    price = Regex.Match(price, "(\\d|\\.)+").Value;

                    string lookedPeopleNum = Regex.Match(houseItems[i], "<div class=\"square\">.*?<span class=\"num\">.*?</span>").Value;
                    lookedPeopleNum = Regex.Match(lookedPeopleNum, "(\\d|\\.)+").Value;

                    MyHouseInfo renthouse = new MyHouseInfo(dataId, name, cityName, region, discription, realMapUrl, layoutMapUrl, detailUrl, configuration, area, totalPrice, price, lookedPeopleNum, url);
                    rentHouseCol.Add(renthouse);        //新建房屋数据对象
                }
                SaveHouseDataToDB(rentHouseCol, "RENTINGHOUSE");    //保存数据到数据库，成功返回True
                return true;
            }
            catch (Exception ex)
            {
                Console.WriteLine(DateTime.Now.ToString("yyyy-MM-dd-HH-mm-ss") + " " + ex.Message + " " + url + " " + cityName);
                return false;       //分析出错，放回False，重新再来一遍
            }
        }
        /// <summary>
        /// //获取小区房价信息
        /// </summary>
        /// <param name="cityName">城市</param>
        /// <param name="content">网址内容</param>
        /// <return>True 成功获取数据，False 出错</return>
        private bool CrawlHoodHousePriceData(string url, string cityName, string region, string content)
        {
            try
            {
                List<MyHouseHoodInfo> hoodHouseCol = new List<MyHouseHoodInfo>();
                string houseInfo = Regex.Match(content, "<ul id=\"house-lst\" class=\"house-lst\".*?</ul>").Value;       //正则表达式分析
                string[] houseItems = Regex.Match(houseInfo, "<li.*?</ul>").Value.Replace("</ul>", "").Split(new string[] { "</li>" }, StringSplitOptions.RemoveEmptyEntries);
                for (int i = 0; i < houseItems.Length; i++)
                {
                    if (houseItems[i].Trim() == "" || houseItems[i].Trim().IndexOf("没有找到") != -1)
                        continue;
                    string detailUrl = Regex.Match(houseItems[i], "href=\".*?\"").Value;
                    detailUrl = detailUrl == "" ? detailUrl : detailUrl.Substring(6, detailUrl.Length - 7);
                    string realMapUrl = Regex.Match(houseItems[i], "src=\".*?\"").Value;
                    realMapUrl = realMapUrl == "" ? realMapUrl : realMapUrl.Substring(5, realMapUrl.Length - 6);

                    string name = Regex.Match(houseItems[i], "title=\".*?\"").Value;
                    name = name == "" ? name : name.Substring(7, name.Length - 8);

                    string avePriceMsg = Regex.Match(houseItems[i], "<div class=\"col-3\">.*?</div>").Value;
                    string avePrice = Regex.Match(avePriceMsg, "<span class=\"num\">.*?</span>").Value;
                    avePrice = Regex.Match(avePrice, "(\\d|\\.)+").Value;

                    string houseNumMsg = Regex.Match(houseItems[i], "<div class=\"col-2\">.*?</div>").Value;
                    string houseNum = Regex.Match(houseNumMsg, "<span class=\"num\">.*?</span>").Value;
                    houseNum = Regex.Match(houseNum, "(\\d|\\.)+").Value;

                    MyHouseHoodInfo hoodHouse = new MyHouseHoodInfo(name, cityName, region, avePrice, houseNum, realMapUrl, detailUrl, url);
                    hoodHouseCol.Add(hoodHouse);         //新建小区数据对象
                }
                SaveHoodHousePriceToDB(hoodHouseCol);    //保存数据到数据库，成功返回True
                return true;
            }
            catch (Exception ex)
            {
                Console.WriteLine(DateTime.Now.ToString("yyyy-MM-dd-HH-mm-ss") + " " + ex.Message + " " + url + " " + cityName);
                return false;       //分析出错，放回False，重新再来一遍
            }
        }
        #endregion

        #region WebServiceMethods
        /// <summary>
        /// //获取指定网址的网页内容
        /// </summary>
        /// <param name="url">网址</param>
        /// <returns>网页内容</returns>
        private string GetUrlHtmlContent(string url)
        {
            while (true)            //持续请求直至请求到，有时候可能出现丢包、延时等现象
            {
                try
                {
                    HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);          //请求网址
                    HttpWebResponse response = (HttpWebResponse)request.GetResponse();
                    byte[] data = new byte[1024];
                    Stream st = response.GetResponseStream();
                    string content = string.Empty;
                    while (true)             //获取网页内容
                    {
                        int bytes = st.Read(data, 0, 1024);
                        content += System.Text.Encoding.UTF8.GetString(data, 0, bytes);
                        if (bytes == 0)
                            break;
                    }
                    content = content.Replace("\n", "").Replace("\t", "").Replace("\r", "");
                    st.Dispose();
                    response.Dispose();
                    return content;
                }
                catch(Exception ex)
                {
                    Console.WriteLine(DateTime.Now.ToString("yyyy-MM-dd-HH-mm-ss") + " " + ex.Message + " " + url);
                    Thread.Sleep(1000);
                }
            }
        }
        #endregion

        #region SaveMethods
        /// <summary>
        /// //保存房屋数据到数据库
        /// </summary>
        /// <param name="HouseDataCol">房屋数据集合</param>
        /// <param name="tableName">表名称</param>
        /// <returns>成功与否</returns>
        private bool SaveHouseDataToDB(List<MyHouseInfo> HouseDataCol,string tableName)
        {
            string commandText = string.Empty;
            for (int i = 0; i < HouseDataCol.Count; i++)
                commandText += "INSERT INTO `" + tableName + "` VALUES(" + HouseDataCol[i].ToString() + ");";     
            if (commandText != string.Empty)
            {
                MySqlConnection mySqlCon = new MySqlConnection(mySqlConText);
                mySqlCon.Open();
                MySqlCommand mySqlCommand = new MySqlCommand(commandText, mySqlCon);
                mySqlCommand.ExecuteNonQuery();
                mySqlCommand.Dispose();
                mySqlCon.Close();
                mySqlCon.Dispose();
            }
            return true;
        }
        /// <summary>
        /// //保存楼盘数据到数据库
        /// </summary>
        /// <param name="BuildingDataCol">楼盘数据集合</param>
        /// <returns>成功与否</returns>
        private bool SaveBuildingDataToDB(List<MyBuildingInfo> BuildingDataCol)
        {
            string commandText = string.Empty;
            for (int i = 0; i < BuildingDataCol.Count; i++)
                commandText += "INSERT INTO `SELLINGHOUSE` VALUES(" + BuildingDataCol[i].ToString() + ");";
            if (commandText != string.Empty)
            {
                MySqlConnection mySqlCon = new MySqlConnection(mySqlConText);
                mySqlCon.Open();
                MySqlCommand mySqlCommand = new MySqlCommand(commandText, mySqlCon);
                mySqlCommand.ExecuteNonQuery();
                mySqlCommand.Dispose();
                mySqlCon.Close();
                mySqlCon.Dispose();
            }
            return true;
        }
        /// <summary>
        /// //保存小区数据到数据库
        /// </summary>
        /// <param name="hoodHouseCollection">小区数据集合</param>
        /// <returns>成功与否</returns>
        private bool SaveHoodHousePriceToDB(List<MyHouseHoodInfo> hoodHouseCollection)
        {
            string commandText = string.Empty;
            for (int i = 0; i < hoodHouseCollection.Count; i++)
                commandText += "INSERT INTO `HOODHOUSE` VALUES(" + hoodHouseCollection[i].ToString() + ");";
            if (commandText != string.Empty)
            {
                MySqlConnection mySqlCon = new MySqlConnection(mySqlConText);
                mySqlCon.Open();
                MySqlCommand mySqlCommand = new MySqlCommand(commandText, mySqlCon);
                mySqlCommand.ExecuteNonQuery();
                mySqlCommand.Dispose();
                mySqlCon.Close();
                mySqlCon.Dispose();
            }
            return true;
        }
        #endregion
    }
}
